package com.chance.cc.crawler.development.bootstrap.dcdapp;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

public class DCDWDStart {
    private static final String domain = "wenda";

    public static void main(String[] args) {
        wenda();
    }

    private static void wenda(){
        String url = "https://www.dongchedi.com/auto/series/1145";
        String brandUrl = "https://www.dongchedi.com/motor/pc/car/series/selected_series?city_name=%EF%BC%9F&series_ids=1145";
        String site = "dcdWenda";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
//                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1627464120000L,1628438400000L}))
                .releaseTime(System.currentTimeMillis())
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(site);
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("brandUrl",brandUrl);
        CrawlerRecord commontRecord = new CrawlerRecord();
        commontRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        commontRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain));
        commontRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1627464120000L,1628438400000L}));//从07.28到8.05
        crawlerRequestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_info", JSON.toJSONString(commontRecord));

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline("kafka")
                //.consoleResultPipeline("redis")
                .fileResultPipeline("kafka","F:\\chance_log\\dcdapp\\kafka.log",false)
                //.fileResultPipeline("redis","F:\\chance_log\\dcdapp\\redis.log",false)
                .requestRecord(crawlerRequestRecord)
                .build("com.chance.cc.crawler.development.scripts.dcdapp")
                .start();
    }
}
